package com.lkl.utils;

import cn.hutool.core.date.DateBetween;
import cn.hutool.core.date.DateTime;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.util.CharsetUtil;
import com.lkl.Condition;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.List;

public class JobProcessor implements PageProcessor {
    final static String[] cssNameList = {"tr.provincetr","tr.citytr","tr.countytr"," tr.towntr","tr.villagetr"};
    //负责解析页面
    @Override
    public void process(Page page) {
        //省 市 、区 、街道/镇 、村/居委会
        //解析数据
        for (String cssName : cssNameList) {
            List<String> all = page.getHtml().css(cssName).all();
            if (all.size() > 0) {
                //存入数据
               ProcessorUtil.getData(page,cssName);
               return;
            }
        }
    }

    private Site site = Site.me();

    @Override
    public Site getSite() {
        return site;
    }
    //主函数

    public static long dataTotalSum = 0; //数据总数
    public static long dataTotal = 0;//省级数据总数
    public static long pageTotalSum = 0; //页面总数
    public static long pageTotal = 0;//省级页面总数
    static String path = FileUtils.filePath + "/sql/log.text";
    public static void main(String[] args) throws InterruptedException {
        //记录开始时间
        DateTime startTime = new DateTime();
        Condition.provinceMap.forEach((k, v) -> {
            try {
                start(k,v);
            } catch (InterruptedException e) {
                System.err.println(e.getMessage());
            }
        });
        System.out.println("全部完成！查询停止");
        DateBetween between = new DateTime().between(startTime);
        String log = "爬取的数据总数：" + dataTotalSum + "，页面总数：" + pageTotalSum + "，程序运行时间：" + between;
        List<String> list1 = new ArrayList<>();
        list1.add(log);
        FileUtil.writeLines(list1,path, CharsetUtil.CHARSET_UTF_8,true);
    }

    private static void start(String code, String name) throws InterruptedException {
        // 重置数据条数和页数
        dataTotal = 0;
        pageTotal = 0;
        //定义文件存储路径
        FileUtils.sqlPath = FileUtils.filePath + "/sql/" + name + ".sql";
        DateTime startTime = new DateTime();
        System.out.println(name + "爬虫启动时间：" + startTime);
        System.out.println("开始爬取数据。。。");
        String str = "insert into `address`(`code`,`name`,`parent_code`) values ";
        List<String> list = new ArrayList<>();
        list.add(str);
        FileUtil.writeLines(list,FileUtils.sqlPath, CharsetUtil.CHARSET_UTF_8,true);
        //官方数据网站，按年份修改，本次年份为2023
        String url = "https://www.stats.gov.cn/sj/tjbz/tjyqhdmhcxhfdm/2023/" + code + ".html";
        Spider.create(new JobProcessor())
                .addUrl(url)  //设置要爬取数据的页面，
                .run(); //执行爬虫
        //每隔一秒检查线程池是否超过两分钟没有写入数据，如果有则结束循环
        boolean close = false;
        while (!close) {
            long diffInMinutes = ChronoUnit.MINUTES.between(
                    LocalDateTime.ofInstant(Instant.ofEpochMilli(FileUtils.startTime), ZoneId.systemDefault()),
                    LocalDateTime.ofInstant(Instant.ofEpochMilli(System.currentTimeMillis()), ZoneId.systemDefault())
            );
            //超过两分钟没有写入数据，则有序关闭线程池
            if (diffInMinutes > 2) {
                System.err.println("已超过两分钟没有写入数据！");
                close = true;
            }
            Thread.sleep(1000);
        }
        System.out.println("爬虫开始终止。。。");
        DateTime endTime = new DateTime();
        System.out.println("终止时间：" + endTime);
        DateBetween between = endTime.between(startTime);
        System.out.println("爬虫运行时间：" + between.toString());
        //统计数据总数
        dataTotalSum += dataTotal;
        pageTotalSum += pageTotal;
        //写入数据
        String log = name + "的数据总数：" + dataTotal + "，页面总数：" + pageTotal + "，爬虫运行时间：" + between;
        List<String> list1 = new ArrayList<>();
        list1.add(log);
        FileUtil.writeLines(list1,path, CharsetUtil.CHARSET_UTF_8,true);
        System.out.println(name + "的线程终止，结束爬虫！\n");
    }
}